import json
from datetime import datetime
from urllib import parse

import requests
from bs4 import BeautifulSoup
from sqlalchemy import Column, String, create_engine, Integer, DateTime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

from splderDetail import spider_detail

header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0',
    'Cookie': 't=0c68246d88d054b0015a9cda35cd1d21; ali_apache_id=11.179.217.87.1614648678331.221451.0; xman_us_f=x_locale=zh_CN&x_l=1&last_popup_time=1614909488392&x_user=CN|jifeng|wang|cnfm|250856201&no_popup_today=n; xman_f=X7QnXGnCqu0J3Q8B26FzIC64EMBoytMy1k/v2HWp325XAwqWcsgwvQzt59r7UZRsP2WJ25YLJT/YEv8CyU5Zl+/HPQHKLYSYhgl5SpgH+1kueNWCb4Yp+XtAZbLEmYB7InXRPSxmL2WrjVZjrHPnz4goVDOlNtIuQfBowKXwjOT+SmMN9S66in+f2dKa2Ri8n7334zGMSLw8sPbUM0x6BnI6F6MrcH32xndeUepVHOKP/rIOaWEX59ayez8IzRvoLkrgBrbB8Khgzp/7e+Df0SlcCNz5XeNPfx8f3o8QTReK5j9ytMcllB7rj+gWGcPpAG1Nu6C6XgbauHls9dzGyQHxwmfQXeDz1i/XI6fLk37kTsj0PPbl4BDcJnIvd7LolSDzzoz/MHeP93Ig5a4xSQ==; _bl_uid=Fkkbblk0rgOctj464v1Le94xj103; cna=J+NIF+8t9B4CAd0P6Ablk8mo; isg=BPz8DAD8sMDRGYRuEwQq-Kcpzpqu9aAfA15CANZ9GOfKoZwr_wTVrgYbhUnZ8th3; l=eBMAz8cVjIYqSAftBOfahurza77O7IOYYuPzaNbMiOCP_4fw5wbAB6N39vTeCn1Vhs_eJ3ydIATJBeYBqC0u0d6Omcgsr4Dmn; tfstk=cpdhBR61YpWBK0MNMX1BRZJN3OdhZ8VPmQR6_QjiZ-eNnHAFi9wa3TsiSM0is81..; history=company%5E%0A200720515%24%0A200705209%24%0A246245818%24%0A213610809; ali_apache_track=ms=|mt=2|mid=cn1540671967kart; intl_common_forever=RmruA81VPEsTG6ZhxTcSZwjxvCWws5lQqTHHkKR1y42rgLxAbf1smw==; _m_h5_tk=2e741ee33ca094f67b3896746489d5a2_1615631209621; _m_h5_tk_enc=5a82b611ea48fe68e6a70927036b4150; XSRF-TOKEN=cec12b20-9883-4561-b50f-446a2b0085c5; ali_apache_tracktmp=W_signed=Y; cookie2=1b842c5e58d955ad21929ed21d1011d5; _tb_token_=336ee337e7ee3; acs_usuc_t=acs_rt=64eb9c70a79943069807fc7482a109a3; xman_t=rfueDChw5DhXf8VPwBIwDNEmyyLMp1AC8QOkEVXagzVhTwK9aRV0jyHKNmNsnx+TsJWSvKIKs3u5i7t2AJuLbdpHg7PKWhhtmWuVfIloaUVGhpXNp0rbmxlZtai0fsAvMOrIFtGG1H/CeeSIKjxiVQ+1uNpEj2SZ8n1VgzCU7DKQFWsPrdAM5igXWQZSCCit2TBR2LMJiYXBuReRHnxjeqvwyjKFLaMYbPaWogp2kMKutRoyr9JSsDHDjZYVm7UsCfFYiojqcRygE9DPR4jNTEhihn9ArzBZeBn5EgVQpAZI5C3N0rX7t4adf5kibBLw6hoCiJ+awbca8yih7BlA8six8u0WhdYZERCuD6pB1b5XFdxoKhF5KyVIAXA/kez4/wRQ6TEOf7zVyc/T9aCJUfnDh4x25eSNhCVgkKS5CvbTihP5rApIpdoIr8UySB0PCZmJZ0uRSDkJ9BEXssPC3Kd6KDUQEvipoRsigBMoqEiOUWhqL+wK+Yrx21m0EzXSEiuoKt8gze8mjrPFq0zNLRKgeFF0KS8dQ8wgvk0kPjPONcQZV0spbqVG7zaDfrhATBIIjG1Mx7ny1x6BWITx+QzcoyLQ2tSIpIQuxlj/TkcXedSyGVAFw2SKEGj2wfTPcBo7a4jFPzOxCBdsKsDXSfBvdlwy9FbKmbRrwsXvaoRj/YAE1PnWj8aillVbrLW4wXvHw6ydahqUiqRc5g+HSMeI1yWg1WMB; JSESSIONID=E80A5C3C5A1EE8080F3A63BBFBF8A271; xlly_s=1; x5sec=7b22736370726f78793b32223a223162653030643838663136383363386466363235343634613465613964333334434979457649494745494c657a2b616d6835435a354145777576337a66773d3d227d; atm-whl=-1%260%260%260; atm-x=__ll%3D-1; _samesite_flag_=true; _hvn_login=4; csg=9a9430e0; xman_us_t=ctoken=ckl7vd8062ao&l_source=alibaba&x_user=+R5cFSnhT7PndTU8dxx6BVpAAUdhm1IZog36+ixNqa8=&x_lid=cn1540671967kart&sign=y&need_popup=y; intl_locale=zh_CN'
    # 'Referer': url

}
main_url = ''

url = 'https://www.alibaba.com/trade/search?fsb=y&IndexArea=company_en&CatId=&SearchText=nanjing'

# 创建对象的基类:
Base = declarative_base()
# 初始化数据库连接:
engine = create_engine('mysql+mysqlconnector://root:yizheng@192.168.0.245:3306/test',
                       echo=True,
                       max_overflow=5)
Base.metadata.create_all(engine)
DBSession = sessionmaker(bind=engine)
session = DBSession()


# 定义对象:
class CpUser(Base):
    # 表的名字:
    __tablename__ = 'nanjing'

    # 表的结构:
    id = Column(Integer, primary_key=True)
    contact_name = Column(String(200))
    Telephone = Column(String(200))
    Address = Column(String(200))
    Zip = Column(String(200))
    CountryRegion = Column(String(200))
    ProvinceState = Column(String(200))
    City = Column(String(200))
    CompanyName = Column(String(200))
    OperationalAddress = Column(String(200))
    Website = Column(String(200))
    Websiteonalibabacom = Column(String(200))
    accountFax = Column(String(200))
    accountMobileNo = Column(String(200))
    accountPhone = Column(String(200))
    time = Column(DateTime, default=datetime.now)


Base.metadata.create_all(engine)

hsession = requests.session()


def spider(url):
    r = hsession.get(url, headers=header)
    soup = BeautifulSoup(r.text, 'lxml')
    data = soup.select(
        '#J-items-content > div > div > div.top > div.corp > div.item-title > div.title-wrap > h2 > a')
    next_page = 'https:' + soup.select('.next')[0].get('href')
    i = 0

    global main_url
    main_url = data[0].get('href')
    # header['Referer'] = main_url
    # contactinfo_rul = main_url[0:main_url.rindex('/') + 1] + 'contactinfo.html'
    # rt = requests.get(contactinfo_rul, header).text

    # parse_detail(rt)
    for d in data:
        main_url = d.get('href')
        # header['Referer'] = main_url
        contactinfo_rul = main_url[0:main_url.rindex('/') + 1] + 'contactinfo.html'
        rt = hsession.get(contactinfo_rul, headers=header).text
        # print(r.cookies.)
        parse_detail(rt)

    while i < 38:
        i += 1
        try:
            spider(next_page)
        except:
            print(url)
            break


def parse_detail(rt):
    soup = BeautifulSoup(rt, 'lxml')

    root = soup.select('#hd > div > div > div')
    try:
        data_dict = json.loads(parse.unquote(root[1].get('module-data')))
        encryptAccountId = data_dict['mds']['moduleData']['data']['encryptAccountId']
    except:
        try:
            data_dict = json.loads(parse.unquote(root[0].get('module-data')))
            encryptAccountId = data_dict['mds']['moduleData']['data']['encryptAccountId']
        except:
            return

    try:
        contact_name = soup.select('.contact-name')[0].get_text()
    except:
        contact_name = ''

    msg = {
        'contact_name': contact_name,
    }

    msg_table = soup.select('.info-table > tr')
    for i in msg_table:
        if i.select('td'):
            msg[i.select('th')[0].get_text()
                .replace(' ', '').replace(':', '').replace('.', '').replace('/', '')] = \
                i.select('td')[0].get_text()

    tt = soup.select('tr.info-item ')
    for i in tt:
        msg[i.select('th > span')[-1].get_text()
            .replace(' ', '').replace(':', '').replace('.', '')] = i.select('td')[0].get_text()

    detail_url = main_url[0:main_url.rindex(
        '/') + 1] + 'event/app/contactPerson/showContactInfo.htm?encryptAccountId=' + encryptAccountId

    if 'https' not in detail_url:
        detail_url = detail_url.replace('http', 'https')

    detail_msg = json.loads(spider_detail(detail_url))
    all_msg = dict(msg, **detail_msg['contactInfo'])
    user = CpUser(contact_name=all_msg.get('contact_name'), Telephone=all_msg.get('Telephone'),
                  Address=all_msg.get('Address'), Zip=all_msg.get('Zip'),
                  CountryRegion=all_msg.get('CountryRegion'),
                  ProvinceState=all_msg.get('ProvinceState'),
                  City=all_msg.get('City'), CompanyName=all_msg.get('CompanyName'),
                  OperationalAddress=all_msg.get('OperationalAddress'),
                  Website=all_msg.get('Website'),
                  Websiteonalibabacom=all_msg.get('Websiteonalibabacom'),
                  accountFax=all_msg.get('accountFax'),
                  accountMobileNo=all_msg.get('accountMobileNo'),
                  accountPhone=all_msg.get('accountPhone'), )
    session.add(user)
    session.commit()


if __name__ == '__main__':
    spider(url)

    # a = {
    #     "contact_name": "Ms. Suzy Tian",
    #     "Telephone": "View details",
    #     "Address": "2nd FL,4TH BUILDING, 2 XIYUANWU ROAD, WESTLAKE TECHNOLOGY GARDEN",
    #     "Zip": "310030",
    #     "CountryRegion": "China",
    #     "ProvinceState": "Zhejiang",
    #     "City": "Hangzhou",
    #     "CompanyName": "Hangzhou Hiland Technology Co., Ltd.",
    #     "OperationalAddress": "Room 803, West 1st Building, Xigang Development Center, Sandun Town, Hangzhou, Zhejiang, China",
    #     "Website": "http://www.hiland.cchttp://michiland.en.made-in-china.com/",
    #     "Websiteonalibabacom": "hiland.en.alibaba.com",
    #     "accountFax": "86-571-81958377",
    #     "accountMobileNo": "13588813115",
    #     "accountPhone": "86-571-81958387"
    # }
    # ks = a.keys()
    # t = 'user = CpUser('
    # for k in ks:
    #     t += k + '=all_msg[\'' + k + '\'],'
    # t += ')'
    # print(t)
